{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "lesbian-springer",
   "metadata": {},
   "source": [
    "## Huggingface Examples\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b1925df",
   "metadata": {},
   "source": [
    "### Text Generation Model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8b4be911",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: text-gen\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/mlserver/huggingface/text-generation\"\r\n",
      "  requirements:\r\n",
      "  - huggingface\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/hf-text-gen.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26c11833",
   "metadata": {},
   "source": [
    "Load the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "checked-cream",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/hf-text-gen.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "interracial-secret",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model status text-gen -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "upper-cholesterol",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "\t\"model_name\": \"text-gen_1\",\r\n",
      "\t\"model_version\": \"1\",\r\n",
      "\t\"id\": \"121ff5f4-1d4a-46d0-9a5e-4cd3b11040df\",\r\n",
      "\t\"parameters\": {},\r\n",
      "\t\"outputs\": [\r\n",
      "\t\t{\r\n",
      "\t\t\t\"name\": \"output\",\r\n",
      "\t\t\t\"shape\": [\r\n",
      "\t\t\t\t1,\r\n",
      "\t\t\t\t1\r\n",
      "\t\t\t],\r\n",
      "\t\t\t\"datatype\": \"BYTES\",\r\n",
      "\t\t\t\"parameters\": {\r\n",
      "\t\t\t\t\"content_type\": \"hg_jsonlist\"\r\n",
      "\t\t\t},\r\n",
      "\t\t\t\"data\": [\r\n",
      "\t\t\t\t\"{\\\"generated_text\\\": \\\"Once upon a time in a galaxy far away, the planet is full of strange little creatures. A very strange combination of creatures in that universe, that is. A strange combination of creatures in that universe, that is. A kind of creature that is\\\"}\"\r\n",
      "\t\t\t]\r\n",
      "\t\t}\r\n",
      "\t]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer text-gen \\\n",
    "  '{\"inputs\": [{\"name\": \"args\",\"shape\": [1],\"datatype\": \"BYTES\",\"data\": [\"Once upon a time in a galaxy far away\"]}]}' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "right-talent",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = !seldon model infer text-gen --inference-mode grpc \\\n",
    "   '{\"inputs\":[{\"name\":\"args\",\"contents\":{\"bytes_contents\":[\"T25jZSB1cG9uIGEgdGltZSBpbiBhIGdhbGF4eSBmYXIgYXdheQo=\"]},\"datatype\":\"BYTES\",\"shape\":[1]}]}' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "4a7d7108",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'{\"generated_text\": \"Once upon a time in a galaxy far away\\\\n\\\\nThe Universe is a big and massive place. How can you feel any of this? Your body doesn\\'t make sense if the Universe is in full swing \\\\u2014 you don\\'t have to remember whether the\"}'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "import base64\n",
    "r = json.loads(res[0])\n",
    "base64.b64decode(r[\"outputs\"][0][\"contents\"][\"bytesContents\"][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d95a5f03",
   "metadata": {},
   "source": [
    "Unload the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "ceramic-illness",
   "metadata": {},
   "outputs": [],
   "source": [
    "!seldon model unload text-gen"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ff63bb7",
   "metadata": {},
   "source": [
    "### Custom Text Generation Model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "6d48e38c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: custom-tiny-stories-text-gen\r\n",
      "spec:\r\n",
      "  storageUri: \"gs://seldon-models/scv2/samples/mlserver_1.3.5/huggingface-text-gen-custom-tiny-stories\"\r\n",
      "  requirements:\r\n",
      "    - huggingface\r\n"
     ]
    }
   ],
   "source": [
    "!cat ./models/hf-text-gen-custom-tiny-stories.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62ec637f",
   "metadata": {},
   "source": [
    "Load the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "abef8088",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/hf-text-gen-custom-tiny-stories.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f25c54d4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model status custom-tiny-stories-text-gen -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "21f26bbd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "\t\"model_name\": \"custom-tiny-stories-text-gen_1\",\r\n",
      "\t\"model_version\": \"1\",\r\n",
      "\t\"id\": \"d0fce59c-76e2-4f81-9711-1c93d08bcbf9\",\r\n",
      "\t\"parameters\": {},\r\n",
      "\t\"outputs\": [\r\n",
      "\t\t{\r\n",
      "\t\t\t\"name\": \"output\",\r\n",
      "\t\t\t\"shape\": [\r\n",
      "\t\t\t\t1,\r\n",
      "\t\t\t\t1\r\n",
      "\t\t\t],\r\n",
      "\t\t\t\"datatype\": \"BYTES\",\r\n",
      "\t\t\t\"parameters\": {\r\n",
      "\t\t\t\t\"content_type\": \"hg_jsonlist\"\r\n",
      "\t\t\t},\r\n",
      "\t\t\t\"data\": [\r\n",
      "\t\t\t\t\"{\\\"generated_text\\\": \\\"Once upon a time in a galaxy far away. It was a very special place to live.\\\\n\\\"}\"\r\n",
      "\t\t\t]\r\n",
      "\t\t}\r\n",
      "\t]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer custom-tiny-stories-text-gen \\\n",
    "  '{\"inputs\": [{\"name\": \"args\",\"shape\": [1],\"datatype\": \"BYTES\",\"data\": [\"Once upon a time in a galaxy far away\"]}]}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "30e8bb29",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = !seldon model infer custom-tiny-stories-text-gen --inference-mode grpc \\\n",
    "   '{\"inputs\":[{\"name\":\"args\",\"contents\":{\"bytes_contents\":[\"T25jZSB1cG9uIGEgdGltZSBpbiBhIGdhbGF4eSBmYXIgYXdheQo=\"]},\"datatype\":\"BYTES\",\"shape\":[1]}]}' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "eab4929b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'{\"generated_text\": \"Once upon a time in a galaxy far away\\\\nOne night, a little girl named Lily went to\"}'"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "import base64\n",
    "r = json.loads(res[0])\n",
    "base64.b64decode(r[\"outputs\"][0][\"contents\"][\"bytesContents\"][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "877e266c",
   "metadata": {},
   "source": [
    "Unload the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "e1829439",
   "metadata": {},
   "outputs": [],
   "source": [
    "!seldon model unload custom-tiny-stories-text-gen"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6b654e6",
   "metadata": {},
   "source": [
    "````\n",
    "As a next step, why not try running a larger-scale model? You can find a definition for one in ./models/hf-text-gen-custom-gpt2.yaml. However, you may need to request and allocate more memory!\n",
    "````"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
